"""
1.html解析器---使用BeautifulSoup4进行Html解析
（1.提取相关词条页面的url   2.提取当前词条的标题和摘录信息）
2.模块---BeautifulSoup4...
3.接口---parser

"""
from bs4 import BeautifulSoup
import re
from architecture_crawl import HtmlDownloader
class HtmlParser(object):

    def parser(self, page_url, html_cont):
        if page_url is None and html_cont is None:
            return None
        soup = BeautifulSoup(html_cont, "html.parser")
        urls = self.get_new_urls(page_url, soup)
        data = self._get_new_data(page_url, soup)
        return urls, data

    def get_new_urls(self, page_url, soup):
        new_urls = set()
        # links = soup.find_all('a', href=re.compile(r'/item/(.*)/\d+') or re.compile(r'/item/(.*)+'))
        links = soup.find_all('a', href=re.compile(r'/item/(.*)+'))
        import urllib.parse
        for link in links:
            full_urls = urllib.parse.urljoin(page_url, link['href'])
            new_urls.add(full_urls)
        return new_urls

    def _get_new_data(self, page_url, soup):
        data ={}
        title = soup.find('dd', class_="lemmaWgt-lemmaTitle-title").find('h1')
        data['url'] = page_url
        data['title'] = title.get_text
        summary = soup.find('div', class_='lemma-summary')
        data['summary'] = summary.get_text
        return data






# obj = HtmlDownloader.HtmlDownloader()
# cont = obj.download("https://baike.baidu.com/item/网络爬虫/5162711?fr=aladdin")
# obj1 = HtmlParser()
# obj1.parser("www", cont)